#ifndef _Tokenizer_H
#define _Tokenizer_H

#include "Token.h"
#include "TokenNode.h"

#include <string>
#include <vector>

using std::vector;
using std::string;

class Tokenizer
{
public:	
	Tokenizer(string const & str);	
	
	typedef vector<Token>::const_iterator const_iterator;
	const_iterator begin() const;
	const_iterator end()   const;	
	
private:	
	vector<Token> tokens;		
    
	void tokenize(string const & str);
};

typedef vector<Token>::const_iterator const_iterator;
void printTokens(const_iterator begin, const_iterator end);

#endif
